{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true,
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "outputs": [],
   "source": [
    "def get_model():\n",
    "    model = tf.keras.Sequential()\n",
    "    model.add(tf.keras.layers.Dense(1,activation='linear',input_dim=784))\n",
    "    model.compile(optimizer=tf.keras.optimizers.RMSprop(lr=0.1),loss='mean_squared_error',metrics=['mae'])\n",
    "    return model"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "outputs": [],
   "source": [
    "(x_train,y_train),(x_test,y_test) = tf.keras.datasets.mnist.load_data()\n",
    "x_train = x_train.reshape((60000,784)).astype('float32') / 255\n",
    "x_test = x_test.reshape((10000,784)).astype('float32') / 255\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "outputs": [],
   "source": [
    "import datetime\n",
    "class MyCustomCallback(tf.keras.callbacks.Callback):\n",
    "    def on_train_batch_begin(self, batch, logs=None):\n",
    "        print('Training: batch {} begins at {}'.format(batch, datetime.datetime.now().time()))\n",
    "    \n",
    "    def one_train_batch_end(self,batch,logs=None):\n",
    "        print('Training: batch {} ends at {}'.format(batch, datetime.datetime.now().time()))\n",
    "\n",
    "    def on_test_batch_begin(self, batch, logs=None):\n",
    "        print('Evaluating: batch {} begins at {}'.format(batch, datetime.datetime.now().time()))\n",
    "\n",
    "    def on_test_batch_end(self, batch, logs=None):\n",
    "        print('Evaluating: batch {} ends at {}'.format(batch, datetime.datetime.now().time()))"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "outputs": [
    {
     "name": "stdout",
     "text": [
      "Training: batch 0 begins at 18:00:43.257000\n",
      "Training: batch 1 begins at 18:00:43.874000\n",
      "Training: batch 2 begins at 18:00:43.896000\n",
      "Training: batch 3 begins at 18:00:43.901000\n",
      "Training: batch 4 begins at 18:00:43.916000\n"
     ],
     "output_type": "stream"
    }
   ],
   "source": [
    "model = get_model()\n",
    "_= model.fit(x_train,y_train,\n",
    "             batch_size=64,\n",
    "             epochs=1,\n",
    "             steps_per_epoch=5,\n",
    "             verbose=0,\n",
    "             callbacks=[MyCustomCallback()])"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "outputs": [
    {
     "name": "stdout",
     "text": [
      "Evaluating: batch 0 begins at 18:01:24.698000\n",
      "Evaluating: batch 0 ends at 18:01:24.974000\n",
      "Evaluating: batch 1 begins at 18:01:24.975000\n",
      "Evaluating: batch 1 ends at 18:01:24.980000\n",
      "Evaluating: batch 2 begins at 18:01:24.982000\n",
      "Evaluating: batch 2 ends at 18:01:24.987000\n",
      "Evaluating: batch 3 begins at 18:01:24.989000\n",
      "Evaluating: batch 3 ends at 18:01:24.995000\n",
      "Evaluating: batch 4 begins at 18:01:24.997000\n",
      "Evaluating: batch 4 ends at 18:01:25.001000\n"
     ],
     "output_type": "stream"
    }
   ],
   "source": [
    "_= model.evaluate(x_test,y_test,batch_size=128,verbose=0,steps=5,\n",
    "                  callbacks=[MyCustomCallback()])"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "outputs": [
    {
     "name": "stdout",
     "text": [
      "For batch 0, loss is   39.38.\n",
      "For batch 1, loss is  907.57.\n",
      "For batch 2, loss is   21.18.\n",
      "For batch 3, loss is    8.33.\n",
      "For batch 4, loss is    8.37.\n",
      "The average loss for epoch 0 is  196.97 and mean absolute error is    8.21.\n",
      "For batch 0, loss is    6.80.\n",
      "For batch 1, loss is    6.40.\n",
      "For batch 2, loss is    5.99.\n",
      "For batch 3, loss is    3.99.\n",
      "For batch 4, loss is    6.34.\n",
      "The average loss for epoch 1 is    5.90 and mean absolute error is    2.04.\n",
      "For batch 0, loss is    3.87.\n",
      "For batch 1, loss is    6.03.\n",
      "For batch 2, loss is    5.36.\n",
      "For batch 3, loss is    6.87.\n",
      "For batch 4, loss is    8.62.\n",
      "The average loss for epoch 2 is    6.15 and mean absolute error is    2.03.\n"
     ],
     "output_type": "stream"
    }
   ],
   "source": [
    "class LossAndErrorPringCallback(tf.keras.callbacks.Callback):\n",
    "    def on_train_batch_end(self, batch, logs=None):\n",
    "        print('For batch {}, loss is {:7.2f}.'.format(batch, logs['loss']))\n",
    "\n",
    "    def on_test_batch_end(self, batch, logs=None):\n",
    "        print('For batch {}, loss is {:7.2f}.'.format(batch, logs['loss']))\n",
    "\n",
    "    def on_epoch_end(self, epoch, logs=None):\n",
    "        print('The average loss for epoch {} is {:7.2f} and mean absolute error is {:7.2f}.'.format(epoch, logs['loss'], logs['mae']))\n",
    "        \n",
    "model = get_model()\n",
    "_ = model.fit(x_train,y_train,\n",
    "              batch_size=64,\n",
    "              steps_per_epoch=5,\n",
    "              epochs=3,\n",
    "              verbose=0,\n",
    "              callbacks=[LossAndErrorPringCallback()])"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "class EarlyStoppingAtMinLoss(tf.keras.callbacks.Callback):\n",
    "    def __init__(self,patience=0):\n",
    "        super(EarlyStoppingAtMinLoss,self).__init__()\n",
    "        self.patience = patience\n",
    "        self.best_weights = None\n",
    "    \n",
    "    def on_train_begin(self,logs=None):\n",
    "        self.wait = 0\n",
    "        self.stopped_epoch = 0\n",
    "        self.best = np.Inf\n",
    "    \n",
    "    def on_epoch_end(self, epoch, logs=None):\n",
    "        current = logs.get('loss')\n",
    "        if np.less(current,self.best):\n",
    "            self.best = current\n",
    "            self.wait = 0\n",
    "            self.best_weights = self.model.get_weights()\n",
    "        else:\n",
    "            self.wait  += 1\n",
    "            if self.wait >= self.patience:\n",
    "                self.stopped_epoch = epoch\n",
    "                self.model.stop_training = True\n",
    "                print(\"导入当前最佳模型\")\n",
    "                self.model.set_weights(self.best_weights)\n",
    "    \n",
    "    def on_train_end(self,logs=None):\n",
    "        if self.stopped_epoch > 0:\n",
    "            print('在%05d: 提前停止训练'% (self.stopped_epoch+1))\n",
    "            \n",
    "    \n",
    "    \n",
    "    \n",
    "\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "outputs": [
    {
     "name": "stdout",
     "text": [
      "For batch 0, loss is   34.58.\n",
      "For batch 1, loss is  972.56.\n",
      "For batch 2, loss is   24.36.\n",
      "For batch 3, loss is   12.19.\n",
      "For batch 4, loss is    9.02.\n",
      "The average loss for epoch 0 is  210.54 and mean absolute error is    8.64.\n",
      "For batch 0, loss is    6.14.\n",
      "For batch 1, loss is    5.92.\n",
      "For batch 2, loss is    4.76.\n",
      "For batch 3, loss is    4.56.\n",
      "For batch 4, loss is    5.23.\n",
      "The average loss for epoch 1 is    5.32 and mean absolute error is    1.89.\n",
      "For batch 0, loss is    4.54.\n",
      "For batch 1, loss is    5.62.\n",
      "For batch 2, loss is    9.10.\n",
      "For batch 3, loss is    7.66.\n",
      "For batch 4, loss is    9.07.\n",
      "The average loss for epoch 2 is    7.20 and mean absolute error is    2.16.\n",
      "导入当前最佳模型\n",
      "在00003: 提前停止训练\n"
     ],
     "output_type": "stream"
    }
   ],
   "source": [
    "model = get_model()\n",
    "_ = model.fit(x_train,y_train,\n",
    "              batch_size=64,\n",
    "              steps_per_epoch=5,\n",
    "              epochs=30,\n",
    "              verbose=0,\n",
    "              callbacks=[LossAndErrorPringCallback(),EarlyStoppingAtMinLoss()])\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "outputs": [],
   "source": [
    "class LearningRateScheduler(tf.keras.callbacks.Callback):\n",
    "    def __init__(self,schedule):\n",
    "        super(LearningRateScheduler,self).__init__()\n",
    "        self.schedule = schedule\n",
    "    \n",
    "    def on_epoch_begin(self, epoch, logs=None):\n",
    "        if not hasattr(self.model.optimizer,'lr'):\n",
    "            raise ValueError(\"Optimizer没有lr参数\")\n",
    "        lr = float(tf.keras.backend.get_value(self.model.optimizer.lr))\n",
    "        \n",
    "        scheduled_lr = self.schedule(epoch,lr)\n",
    "        tf.keras.backend.set_value(self.model.optimizer.lr,scheduled_lr)\n",
    "        print('Epoch %05d: 学习率为%6.4f.'%(epoch, scheduled_lr))\n",
    "\n",
    "\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "outputs": [
    {
     "name": "stdout",
     "text": [
      "Epoch 00000: 学习率为0.1000.\n",
      "For batch 0, loss is   26.68.\n",
      "For batch 1, loss is  980.97.\n",
      "For batch 2, loss is   25.78.\n",
      "For batch 3, loss is    8.23.\n",
      "For batch 4, loss is    6.68.\n",
      "The average loss for epoch 0 is  209.67 and mean absolute error is    8.38.\n",
      "Epoch 00001: 学习率为0.1000.\n",
      "For batch 0, loss is    6.11.\n",
      "For batch 1, loss is    4.59.\n",
      "For batch 2, loss is    8.20.\n",
      "For batch 3, loss is    6.16.\n",
      "For batch 4, loss is    7.13.\n",
      "The average loss for epoch 1 is    6.44 and mean absolute error is    2.10.\n",
      "Epoch 00002: 学习率为0.1000.\n",
      "For batch 0, loss is    4.27.\n",
      "For batch 1, loss is    5.59.\n",
      "For batch 2, loss is    5.11.\n",
      "For batch 3, loss is    5.81.\n",
      "For batch 4, loss is    6.03.\n",
      "The average loss for epoch 2 is    5.36 and mean absolute error is    1.87.\n",
      "Epoch 00003: 学习率为0.0500.\n",
      "For batch 0, loss is    6.48.\n",
      "For batch 1, loss is    4.87.\n",
      "For batch 2, loss is    4.06.\n",
      "For batch 3, loss is    4.58.\n",
      "For batch 4, loss is    3.51.\n",
      "The average loss for epoch 3 is    4.70 and mean absolute error is    1.68.\n",
      "Epoch 00004: 学习率为0.0500.\n",
      "For batch 0, loss is    5.40.\n",
      "For batch 1, loss is    4.66.\n",
      "For batch 2, loss is    3.49.\n",
      "For batch 3, loss is    3.40.\n",
      "For batch 4, loss is    5.62.\n",
      "The average loss for epoch 4 is    4.52 and mean absolute error is    1.67.\n",
      "Epoch 00005: 学习率为0.0500.\n",
      "For batch 0, loss is    4.83.\n",
      "For batch 1, loss is    3.89.\n",
      "For batch 2, loss is    4.27.\n",
      "For batch 3, loss is    5.05.\n",
      "For batch 4, loss is    3.09.\n",
      "The average loss for epoch 5 is    4.23 and mean absolute error is    1.62.\n",
      "Epoch 00006: 学习率为0.0100.\n",
      "For batch 0, loss is    5.28.\n",
      "For batch 1, loss is    4.81.\n",
      "For batch 2, loss is    3.50.\n",
      "For batch 3, loss is    4.06.\n",
      "For batch 4, loss is    3.85.\n",
      "The average loss for epoch 6 is    4.30 and mean absolute error is    1.67.\n",
      "Epoch 00007: 学习率为0.0100.\n",
      "For batch 0, loss is    3.71.\n",
      "For batch 1, loss is    5.53.\n",
      "For batch 2, loss is    2.70.\n",
      "For batch 3, loss is    3.25.\n",
      "For batch 4, loss is    4.34.\n",
      "The average loss for epoch 7 is    3.90 and mean absolute error is    1.59.\n",
      "Epoch 00008: 学习率为0.0100.\n",
      "For batch 0, loss is    5.61.\n",
      "For batch 1, loss is    3.23.\n",
      "For batch 2, loss is    3.85.\n",
      "For batch 3, loss is    4.94.\n",
      "For batch 4, loss is    4.33.\n",
      "The average loss for epoch 8 is    4.39 and mean absolute error is    1.66.\n",
      "Epoch 00009: 学习率为0.0050.\n",
      "For batch 0, loss is    3.26.\n",
      "For batch 1, loss is    2.90.\n",
      "For batch 2, loss is    3.98.\n",
      "For batch 3, loss is    4.02.\n",
      "For batch 4, loss is    3.33.\n",
      "The average loss for epoch 9 is    3.50 and mean absolute error is    1.53.\n",
      "Epoch 00010: 学习率为0.0050.\n",
      "For batch 0, loss is    3.26.\n",
      "For batch 1, loss is    4.89.\n",
      "For batch 2, loss is    3.25.\n",
      "For batch 3, loss is    4.35.\n",
      "For batch 4, loss is    3.48.\n",
      "The average loss for epoch 10 is    3.84 and mean absolute error is    1.59.\n",
      "Epoch 00011: 学习率为0.0050.\n",
      "For batch 0, loss is    3.37.\n",
      "For batch 1, loss is    4.84.\n",
      "For batch 2, loss is    4.28.\n",
      "For batch 3, loss is    2.86.\n",
      "For batch 4, loss is    4.89.\n",
      "The average loss for epoch 11 is    4.05 and mean absolute error is    1.60.\n",
      "Epoch 00012: 学习率为0.0010.\n",
      "For batch 0, loss is    3.97.\n",
      "For batch 1, loss is    3.60.\n",
      "For batch 2, loss is    3.55.\n",
      "For batch 3, loss is    3.33.\n",
      "For batch 4, loss is    3.69.\n",
      "The average loss for epoch 12 is    3.63 and mean absolute error is    1.53.\n",
      "Epoch 00013: 学习率为0.0010.\n",
      "For batch 0, loss is    5.91.\n",
      "For batch 1, loss is    3.04.\n",
      "For batch 2, loss is    4.42.\n",
      "For batch 3, loss is    3.84.\n",
      "For batch 4, loss is    3.50.\n",
      "The average loss for epoch 13 is    4.14 and mean absolute error is    1.60.\n",
      "Epoch 00014: 学习率为0.0010.\n",
      "For batch 0, loss is    3.96.\n",
      "For batch 1, loss is    4.02.\n",
      "For batch 2, loss is    4.23.\n",
      "For batch 3, loss is    4.57.\n",
      "For batch 4, loss is    5.87.\n",
      "The average loss for epoch 14 is    4.53 and mean absolute error is    1.67.\n"
     ],
     "output_type": "stream"
    }
   ],
   "source": [
    "LR_CHEDULE = [\n",
    "    (3, 0.05), (6, 0.01), (9, 0.005), (12, 0.001)\n",
    "]\n",
    "\n",
    "\n",
    "def lr_schedule(epoch,lr):\n",
    "    if epoch < LR_CHEDULE[0][0] or epoch > LR_CHEDULE[-1][0]:\n",
    "        return lr\n",
    "    \n",
    "    for i in range(len(LR_CHEDULE)):\n",
    "        if epoch == LR_CHEDULE[i][0]:\n",
    "            return LR_CHEDULE[i][1]\n",
    "    return lr\n",
    "\n",
    "model = get_model()\n",
    "_ = model.fit(x_train,y_train,\n",
    "              batch_size=64,\n",
    "              steps_per_epoch=5,\n",
    "              epochs=15,\n",
    "              verbose=0,\n",
    "              callbacks=[LossAndErrorPringCallback(),LearningRateScheduler(lr_schedule)])\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n",
     "is_executing": false
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  },
  "pycharm": {
   "stem_cell": {
    "cell_type": "raw",
    "source": [],
    "metadata": {
     "collapsed": false
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}